{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 第二步：调整树的参数：max_depth & min_child_weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "\n",
    "import xgboost as xgb\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from matplotlib import pyplot\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "import seaborn as sns\n",
    "\n",
    "from numpy import nan as NaN\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Filled_Form_N</th>\n",
       "      <th>Filled_Form_Y</th>\n",
       "      <th>Device_Type_Mobile</th>\n",
       "      <th>Device_Type_Web-browser</th>\n",
       "      <th>Mobile_Verified_N</th>\n",
       "      <th>Mobile_Verified_Y</th>\n",
       "      <th>Source_S122</th>\n",
       "      <th>Source_S123</th>\n",
       "      <th>Source_S124</th>\n",
       "      <th>Source_S127</th>\n",
       "      <th>...</th>\n",
       "      <th>Loan_Amount_Submitted</th>\n",
       "      <th>Loan_Tenure_Submitted</th>\n",
       "      <th>Interest_Rate</th>\n",
       "      <th>Processing_Fee</th>\n",
       "      <th>Disbursed</th>\n",
       "      <th>DOB_month</th>\n",
       "      <th>DOB_year</th>\n",
       "      <th>age</th>\n",
       "      <th>Lead_Creation_Date_month</th>\n",
       "      <th>Lead_Creation_Date_year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>620000.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>13.99</td>\n",
       "      <td>3100.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8</td>\n",
       "      <td>1987</td>\n",
       "      <td>32</td>\n",
       "      <td>7</td>\n",
       "      <td>2015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>260000.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>33.00</td>\n",
       "      <td>2000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "      <td>1993</td>\n",
       "      <td>26</td>\n",
       "      <td>7</td>\n",
       "      <td>2015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>100000.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>28.50</td>\n",
       "      <td>6600.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "      <td>1978</td>\n",
       "      <td>41</td>\n",
       "      <td>7</td>\n",
       "      <td>2015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>200000.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>28.50</td>\n",
       "      <td>5000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6</td>\n",
       "      <td>1985</td>\n",
       "      <td>34</td>\n",
       "      <td>7</td>\n",
       "      <td>2015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>300000.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>16.25</td>\n",
       "      <td>7000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4</td>\n",
       "      <td>1975</td>\n",
       "      <td>44</td>\n",
       "      <td>7</td>\n",
       "      <td>2015</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 43 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Filled_Form_N  Filled_Form_Y  Device_Type_Mobile  Device_Type_Web-browser  \\\n",
       "0              0              1                   1                        0   \n",
       "1              1              0                   1                        0   \n",
       "2              1              0                   0                        1   \n",
       "3              1              0                   0                        1   \n",
       "4              1              0                   0                        1   \n",
       "\n",
       "   Mobile_Verified_N  Mobile_Verified_Y  Source_S122  Source_S123  \\\n",
       "0                  0                  1            1            0   \n",
       "1                  0                  1            1            0   \n",
       "2                  0                  1            1            0   \n",
       "3                  1                  0            1            0   \n",
       "4                  0                  1            1            0   \n",
       "\n",
       "   Source_S124  Source_S127           ...             Loan_Amount_Submitted  \\\n",
       "0            0            0           ...                          620000.0   \n",
       "1            0            0           ...                          260000.0   \n",
       "2            0            0           ...                          100000.0   \n",
       "3            0            0           ...                          200000.0   \n",
       "4            0            0           ...                          300000.0   \n",
       "\n",
       "   Loan_Tenure_Submitted  Interest_Rate  Processing_Fee  Disbursed  DOB_month  \\\n",
       "0                    4.0          13.99          3100.0        0.0          8   \n",
       "1                    4.0          33.00          2000.0        0.0          2   \n",
       "2                    5.0          28.50          6600.0        0.0          2   \n",
       "3                    3.0          28.50          5000.0        0.0          6   \n",
       "4                    5.0          16.25          7000.0        0.0          4   \n",
       "\n",
       "   DOB_year  age  Lead_Creation_Date_month  Lead_Creation_Date_year  \n",
       "0      1987   32                         7                     2015  \n",
       "1      1993   26                         7                     2015  \n",
       "2      1978   41                         7                     2015  \n",
       "3      1985   34                         7                     2015  \n",
       "4      1975   44                         7                     2015  \n",
       "\n",
       "[5 rows x 43 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv('FE_X_train.csv')\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop ids and get labels\n",
    "y_train = train['Disbursed']\n",
    "\n",
    "train = train.drop([\"Disbursed\"], axis=1)\n",
    "X_train = np.array(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': [1, 2, 3], 'min_child_weight': [0.001, 0.01, 0.1]}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#max_depth 3-10， min_child_weight 1-6\n",
    "max_depth = [1,2,3]\n",
    "min_child_weight = [0.001,0.01,0.1]\n",
    "param_test2_2 = dict(max_depth=max_depth, min_child_weight=min_child_weight)\n",
    "param_test2_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(make_scorer(log_loss, greater_is_better=False, needs_proba=True),\n",
       " {'max_depth': 2, 'min_child_weight': 0.001},\n",
       " -0.251843451314537)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# prepare cross validation\n",
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=3)\n",
    "\n",
    "xgb2_2 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=14,  #第一轮参数调整得到的n_estimators最优值\n",
    "        max_depth=2,\n",
    "        min_child_weight=0.001,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'binary:logistic',\n",
    "#         num_class = 9,\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch2_2 = GridSearchCV(xgb2_2, param_grid = param_test2_2, scoring='neg_log_loss',n_jobs=2, cv=kfold)\n",
    "gsearch2_2.fit(X_train , y_train)\n",
    "\n",
    "gsearch2_2.scorer_, gsearch2_2.best_params_, gsearch2_2.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:125: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:125: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:125: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:125: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:125: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:125: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:125: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([0.0510838 , 0.05058174, 0.04910583, 0.06409311, 0.07476077,\n",
       "        0.07838602, 0.07678504, 0.07086678, 0.07124076]),\n",
       " 'std_fit_time': array([0.00236633, 0.00277902, 0.00269141, 0.00224458, 0.01031554,\n",
       "        0.00641789, 0.0053867 , 0.00163144, 0.00237252]),\n",
       " 'mean_score_time': array([0.00403466, 0.0025713 , 0.00280943, 0.00317202, 0.00411034,\n",
       "        0.00452867, 0.00307727, 0.00263677, 0.00260582]),\n",
       " 'std_score_time': array([9.48831332e-04, 2.86258353e-04, 2.31063479e-04, 5.52664194e-04,\n",
       "        1.16907343e-03, 1.62058037e-03, 6.79263863e-04, 4.95150332e-05,\n",
       "        4.58623654e-05]),\n",
       " 'param_max_depth': masked_array(data=[1, 1, 1, 2, 2, 2, 3, 3, 3],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_min_child_weight': masked_array(data=[0.001, 0.01, 0.1, 0.001, 0.01, 0.1, 0.001, 0.01, 0.1],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'max_depth': 1, 'min_child_weight': 0.001},\n",
       "  {'max_depth': 1, 'min_child_weight': 0.01},\n",
       "  {'max_depth': 1, 'min_child_weight': 0.1},\n",
       "  {'max_depth': 2, 'min_child_weight': 0.001},\n",
       "  {'max_depth': 2, 'min_child_weight': 0.01},\n",
       "  {'max_depth': 2, 'min_child_weight': 0.1},\n",
       "  {'max_depth': 3, 'min_child_weight': 0.001},\n",
       "  {'max_depth': 3, 'min_child_weight': 0.01},\n",
       "  {'max_depth': 3, 'min_child_weight': 0.1}],\n",
       " 'split0_test_score': array([-0.25136182, -0.25136182, -0.25136182, -0.24886082, -0.24886082,\n",
       "        -0.24886082, -0.24892153, -0.24892153, -0.24892153]),\n",
       " 'split1_test_score': array([-0.25395572, -0.25395572, -0.25395572, -0.24947394, -0.24947394,\n",
       "        -0.24947394, -0.24920154, -0.24920154, -0.24920154]),\n",
       " 'split2_test_score': array([-0.25423098, -0.25423098, -0.25423098, -0.25366797, -0.25366797,\n",
       "        -0.25366797, -0.25421304, -0.25421304, -0.25421304]),\n",
       " 'split3_test_score': array([-0.25670211, -0.25670211, -0.25670211, -0.25524608, -0.25524608,\n",
       "        -0.25524608, -0.25497486, -0.25497486, -0.25497486]),\n",
       " 'split4_test_score': array([-0.25601938, -0.25601938, -0.25601938, -0.25196869, -0.25196869,\n",
       "        -0.25196869, -0.25278029, -0.25278029, -0.25278029]),\n",
       " 'mean_test_score': array([-0.25445337, -0.25445337, -0.25445337, -0.25184345, -0.25184345,\n",
       "        -0.25184345, -0.25201794, -0.25201794, -0.25201794]),\n",
       " 'std_test_score': array([0.00186366, 0.00186366, 0.00186366, 0.00242672, 0.00242672,\n",
       "        0.00242672, 0.00251692, 0.00251692, 0.00251692]),\n",
       " 'rank_test_score': array([7, 7, 7, 1, 1, 1, 4, 4, 4], dtype=int32),\n",
       " 'split0_train_score': array([-0.25394534, -0.25394534, -0.25394534, -0.25126887, -0.25126887,\n",
       "        -0.25126887, -0.24832647, -0.24832647, -0.24832647]),\n",
       " 'split1_train_score': array([-0.25419667, -0.25419667, -0.25419667, -0.25056432, -0.25056432,\n",
       "        -0.25056432, -0.24771745, -0.24771745, -0.24771745]),\n",
       " 'split2_train_score': array([-0.25341487, -0.25341487, -0.25341487, -0.25015567, -0.25015567,\n",
       "        -0.25015567, -0.24643221, -0.24643221, -0.24643221]),\n",
       " 'split3_train_score': array([-0.25222009, -0.25222009, -0.25222009, -0.24836111, -0.24836111,\n",
       "        -0.24836111, -0.24447224, -0.24447224, -0.24447224]),\n",
       " 'split4_train_score': array([-0.25406395, -0.25406395, -0.25406395, -0.24915502, -0.24915502,\n",
       "        -0.24915502, -0.24682133, -0.24682133, -0.24682133]),\n",
       " 'mean_train_score': array([-0.25356818, -0.25356818, -0.25356818, -0.249901  , -0.249901  ,\n",
       "        -0.249901  , -0.24675394, -0.24675394, -0.24675394]),\n",
       " 'std_train_score': array([0.00072442, 0.00072442, 0.00072442, 0.00102992, 0.00102992,\n",
       "        0.00102992, 0.00132026, 0.00132026, 0.00132026])}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch2_2.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # summarize results\n",
    "# print(\"Best: %f using %s\" % (gsearch1.best_score_, gsearch1.best_params_))\n",
    "# test_means = gsearch2_2.cv_results_[ 'mean_test_score' ]\n",
    "# test_stds = gsearch2_2.cv_results_[ 'std_test_score' ]\n",
    "# train_means = gsearch2_2.cv_results_[ 'mean_train_score' ]\n",
    "# train_stds = gsearch2_2.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "# pd.DataFrame(gsearch2_2.cv_results_).to_csv('my_preds_maxdepth_min_child_weights_2.csv')\n",
    "\n",
    "# # plot results\n",
    "# test_scores = np.array(test_means).reshape(len(min_child_weight), len(max_depth))\n",
    "# train_scores = np.array(train_means).reshape(len(min_child_weight), len(max_depth))\n",
    "\n",
    "# for i, value in enumerate(min_child_weight):\n",
    "#     pyplot.plot(max_depth, test_scores[i], label= 'test_min_child_weight:'   + str(value))\n",
    "# #for i, value in enumerate(min_child_weight):\n",
    "# #    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "# pyplot.legend()\n",
    "# pyplot.xlabel( 'max_depth' )                                                                                                      \n",
    "# pyplot.ylabel( '- Log Loss' )\n",
    "# pyplot.savefig( 'max_depth_vs_min_child_weght2.png' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.251843451314537"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch2_2.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': 2, 'min_child_weight': 0.001}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch2_2.best_params_"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
